import scrapy
from myapp.items import MyappItem
import random
import time


class HotspiderSpider(scrapy.Spider):
    name = 'hotspider'
    start_urls = [
        'https://www.bilibili.com/v/popular/rank/all'
    ]

    def parse(self, response):
        scores = response.xpath('//ul[@class="rank-list"]/li//div[@class="pts"]/div/text()').getall()
        urls = response.xpath('//ul[@class="rank-list"]/li//div[@class="info"]/a/@href').getall()
        # print(len(urls))
        urls = ['https:' + i for i in urls]
        for i in range(len(urls)):
            info = []
            next_url = "https://api.bilibili.com/x/web-interface/view?bvid={}".format(urls[i].split("/")[-1])
            info.append("'" + str(scores[i]) + "'")
            info.append("'" + str(urls[i]) + "'")
            info.append("'" + str(response.url).split("/")[-1].split("?")[0] + "'")
            print(next_url)
            rt = random.randint(1, 5)
            time.sleep(rt)
            yield scrapy.Request(next_url, callback=self.parse2, meta={'info': info})

    def parse2(self, response):
        data = response.json()['data']
        info = response.meta['info']
        info.append("'" + str(data['aid']) + "'")
        info.append("'" + str(data['pic']) + "'")
        info.append("'" + str(data['title']) + "'")
        info.append("'" + str(data['desc']).replace("\n", "") + "'")
        info.append("'" + str(data['owner']['mid']) + "'")
        info.append("'" + str(data['owner']['name']) + "'")
        info.append("'" + str(data['owner']['face']) + "'")
        info.append("'" + str(data['stat']['view']) + "'")
        info.append("'" + str(data['stat']['danmaku']) + "'")
        info.append("'" + str(data['stat']['reply']) + "'")
        info.append("'" + str(data['stat']['favorite']) + "'")
        info.append("'" + str(data['stat']['coin']) + "'")
        info.append("'" + str(data['stat']['share']) + "'")
        info.append("'" + str(data['stat']['like']) + "'")
        info.append("'" + str(data['cid']) + "'")
        response.meta['info'] = info
        next_url = 'https://api.bilibili.com/x/web-interface/card?jsonp=jsonp&mid={}'.format(str(data['owner']['mid']))
        yield scrapy.Request(next_url, callback=self.get_fans, meta={'info': response.meta['info']})


    def get_fans(self, response):
        item = MyappItem()
        # print(response.meta["info"])
        data = response.json()['data']
        follower = "'" + str(data['follower']) + "'"
        info = response.meta['info']
        info.append(follower)
        response.meta['info'] = info
        # print(data['follower'])
        item['info'] = response.meta['info']
        print(response.meta['info'])
        yield item